var MongoClient = require('mongodb').MongoClient, assert = require('assert');
var connection = 'mongodb://120.55.126.201:27017/66xiaoqu';
var async = require('async')
var ObjectId = require('mongodb').ObjectID;

function removeDuplications(collections, uniqueKey){
    var counter = 0
    MongoClient.connect(connection,
        function (err, db) {
            async.eachSeries(collections, function (collection, callbackOuter) {
                counter ++
                //collections.forEach(function (collection) {
                var docIds = {}
                console.log("remove duplicates from %s...", collection)
                var mongoColl = db.collection(collection);
                var filters = {'source':'sofang','commId':{'$exists':true}}
                //var filters = {'source':'sofang','commId':78597}
                mongoColl.find(filters).toArray(function (err, docs) {
                    var docLen = docs.length
                    console.log(docLen)
                    if(docLen == 0){
                        console.log("NO DATA IN %s", collection)
                        callbackOuter()
                        if(counter == collections.length){
                            console.log("FINISHED")
                            db.close()
                            process.exit()
                        }
                        return
                    }
                    docs.forEach(function (doc, i) {
                        var uKey = doc[uniqueKey]
                        if (uKey != null && uKey in docIds) {
                            mongoColl.deleteOne({'_id': doc._id})
                            console.log("removed %s (%s)", doc._id, collection)
                        }
                        docIds[uKey] = 1
                        if (i == docLen - 1) {
                            console.log("All %s duplicates removed\n", collection)
                            callbackOuter()
                            if(counter == collections.length){
                                console.log("FINISHED")
                                //db.close()
                                //process.exit()
                            }
                        }
                    })
                })
            })
        });
}

function getAnjukeCollections(){
    var cities = 'anshan anyang anqing ankang anshun beijing baoding baotou binzhou baoji bengbu benxi beihai baicheng baiyin bayannaoer bayinguoleng bazhong bozhou chengdu chongqing cs cc cz cangzhou changji chifeng changde chenzhou chengde changzhi chizhou chuzhou chaohu chaoyang chaozhou chuxiong dalian dg deyang dali dezhou dongying daqing dandong datong dazhou eerduosi enshi ezhou fz foshan fuyang fushun fuxin fuzhoushi guangzhou gy guilin ganzhou guangan guigang guangyuan hangzhou heb hf haikou huhehaote handan huizhou huanggang huainan huangshan hebi hengyang huzhou hengshui hanzhong huaian huangshi heze huaihua huaibei huludao heyuan hami hegang honghe hulunbeier jinan jilin jx jiangmen jingmen jinzhou jingdezhen jian jining jinhua jieyang jinzhong jiujiang jiaozuo jincheng jingzhou jiamusi jiuquan jixi jiyuan km ks kaifeng kashi kelamayi lanzhou langfang liuzhou luoyang laiwu luan luzhou lijiang linyi liaocheng lianyungang lishui loudi leshan liaoyang lasa linfen longyan luohe liangshan liaoyuan liupanshui mianyang maoming maanshan mudanjiang meishan meizhou nanjing nc nanning nb nantong nanchong nanyang ningde neijiang nanping panzhihua panjin pingdingsha pingxiang puyang putian puer qd qinhuangdao quanzhou qujing qiqihaer quzhou qingyuan qingyang qinzhou rizhao shanghai shenzhen suzhou sy sanya sjz shaoxing shantou shiyan sanmenxia sanming shaoguan shangqiu suqian suihua shaoyang suining shangrao siping shanwei shihezi shizuishan shunde shuozhou shuyang songyuan suizhou suzhoushi tianjin ty tangshan taizhou taian taiz tieling tongliao tongling tianshui tonghua wuhan wuxi weihai weifang wulumuqi wenzhou wuhu wuzhou weinan wenshan wuhai wulanchabu wuwei xm xa xuzhou xiangtan xiangyang xinxiang xinyang xianyang xingtai xiaogan xining xuchang xinzhou xianning xinganmeng xinyu xuancheng yinchuan yangzhou yt yichang yangjiang yongzhou yulinshi yancheng yueyang yuncheng yichun yingkou yulin yibin yiyang yiwu yanan yanbian yangquan yili yingtan yufu yuxi zhengzhou zh zs zhenjiang zibo zhangjiakou zhuzhou zhangzhou zhanjiang zhaoqing zaozhuang zhoushan zunyi zhumadian zigong zhangjiajie zhangqiu zhoukou zhucheng ziyang'.split(' ')
    var anj_collections = []
    cities.forEach(function(city){
        anj_collections.push("xq_anjukeDetail_" + city)
    })
    return anj_collections
}
function getSofangCollections(){
    var cities = 'aq ay ab anshun al ak aks altd aomen alsm as bj bd bb bozhou bz bh baise bazhong bt bijie bs baoji by bycem betl bygl bazhou bx baishan bc chengde chuzhou cangzhou ch chizhou changzhi cs changde chenzhou chaozhou chongzuo cq cd cxyz chifeng changdu cjhz changshu cy cc cz dy dz dt dg deyang dazhou dlbz dh diqing dx dl dd dq dxal ez estjzmz erds fy fz fuzhou fs fcg fushun fx ganzhou gz gl guigang guangyuan ga ganzi gy gnzz guoluo guyuan hd hn huaibei huangshan hs heze hb hshi hg hy hh huizhou heyuan hezhou hc haikou hu honghe hanzhong hlbe haidong haibei huangnan hnzz hx hm ht hld hrb hegang heihe ha hz huzhou hf jdz jj ja jn jining jiaozuo jingmen jingzhou jincheng jz jm jy jyg jinchang jq jiangyin jinzhou jl jixi jms jx jh kaifeng km klmy kzls ksd kunshan la lf ly lw linyi lc luoyang luohe ld liuzhou linfen lb luzhou lvliang ls liangshan lps lj lincang lasa linzhi lz ln lx liaoyang liaoyuan lyg lishui mas mm mz my ms mdj np nc ny nn neijiang nanchong nujiang nqd ningde nj nt nb pt px pds puyang panzhihua pl pj qhd qz qd qingyuan qinzhou qxn qdn qn qj qingyang qqhr qth quzhou rizhao rkzd sjz suzhou sm sr smx sq shiyan suizhou shaoyang shuozhou sg sz st sw sanya suining simao shannandi sl szs shz sy sp songyuan suihua sh su suqian sx tj ts tongling ty ta tr tongliao tc tianshui tlfd tcd tw taicang tl th taizhou tz wuhu wf weihai wh wuzhou wuhai ws wn wuwei wuzhong wlmq wlcb wlcbm wx wz wenchang xt xuancheng xm xinyu xx xc xy xiaogan xianning xiangtan xiangxi xinzhou xa xianyang xn xam xg xiangyang xl xsbn xz yingtan yichun yt yq yc yy yiyang yongzhou yuncheng yj yf yulin yb ya yx yanan yl ys yinchuan yili yiwu yk yanbian yashan yich yancheng yz zjk zhangzhou zb zaozhuang zz zk zmd zhuzhou zjj zh zhanjiang zq zs zg zy zunyi zt zhangye zw zjw zj zhoushan'.split(' ')
    var sf_collections = []
    cities.forEach(function(city){
        sf_collections.push("xq_sfDetail_" + city)
    })
    return sf_collections
}

var anj_cities = getAnjukeCollections()
var sf_cities = getSofangCollections()

var cities = sf_cities
//removeDuplications(cities,'commId')
removeDuplications(['community'],'commId')

//removeDuplications(['_addressrule'],'ZDZQC')